# coding=utf-8

import sys
from scrapy.linkextractors.sgml import SgmlLinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait

from scrapy_sinahealth.items import ScrapyHealthItem

reload(sys);
sys.setdefaultencoding("utf-8");

class HealthSinaSpider(CrawlSpider):
    # 爬虫名称
    name = "sinahealth";
    # 设置下载延时
    download_delay = 3;
    def __init__(self):
        super(HealthSinaSpider, self).__init__()
        self.driver = webdriver.Firefox();
        self.driver.set_page_load_timeout(5);
    pass

    allowed_domains = ["health.sina.com.cn"];
    # 开始URL
    start_urls = ["http://health.sina.com.cn/disease/", "http://health.sina.com.cn/healthcare/"];
    rules = (
        Rule(SgmlLinkExtractor(allow=(r'http://health.sina.com.cn/d/.+/doc-ify.+.shtml',)), callback='parse_content'),
        Rule(SgmlLinkExtractor(allow=(r'http://health.sina.com.cn/hc/.+/doc-ify.+.shtml',)), callback='parse_content'),
    );
    #http://health.sina.com.cn/d/2017-12-23/doc-ifypxrpp3470833.shtml
    #http://health.sina.com.cn/d/2017-12-23/doc-ifypxrpp3212741.shtml
    def parse_content(self, response):
        self.driver.get(response.url);
        wait = WebDriverWait(self.driver, 2);

        item = ScrapyHealthItem();

        title = response.selector.xpath('//h1[@id="main_title"]')[0].extract().decode('utf-8');
        item['title'] = title;

        content = response.selector.xpath('//div[@id="artibody"]')[0].extract().decode('utf-8');
        item['content'] = content;

        url = response.url.strip();
        item['url'] = url;

        issue_date = response.selector.xpath('//span[@class="titer"]')[0].extract().decode('utf-8');
        item['issue_date'] = issue_date;

        source = response.selector.xpath('//span[@class="source"]')[0].extract().decode('utf-8');
        item['source'] = source;

        yield item